/*
 * Copyright (c) Huawei Technologies Co., Ltd. 2020-2022. All rights reserved.
 * Description: TbeVectorReduce类的实现文件
 * Author: qingjicheng
 * Create: 2020-12-1
 */

#include "tbe_vector_reduce.h"
#include <fstream>
#include <dlfcn.h>
#include <unistd.h>
#include <nlohmann/json.hpp>
#include <cstring>
#include <hccl/hccl_types.h>
#include <iostream>
#include <iomanip>
#include <mutex>
#include "runtime/rt.h"
#include "op_json_info.h"
#include "op_tiling.h"
#include "vector_tiling.h"
#include "eletwise_v2.h"
#include "hccl_dl.h"
#include "adapter_rts.h"
#include "adapter_error_manager.h"
#include "device_capacity.h"
#include "mmpa_api.h"
#ifndef HCCD
#include "platform/platform_info.h"
#include "exe_graph/runtime/tiling_context.h"
#include "exe_graph/runtime/tiling_parse_context.h"
#include "exe_graph/lowering/tiling_context_builder.h"
#include "exe_graph/lowering/tiling_parse_context_builder.h"
#include "graph/utils/op_desc_utils.h"
#include "graph/operator_factory.h"
#include "graph/operator_reg.h"
#include "graph/operator.h"
#include "vector_tiling_rt2.h"
#include "graph/ge_tensor.h"
#include "graph/op_desc.h"
#include "graph/utils/graph_utils.h"

namespace ge {
REG_OP(Add)
    .INPUT(x1, TensorType({ DT_INT64 }))
    .INPUT(x2, TensorType({ DT_INT64 }))
    .OUTPUT(y, TensorType({ DT_INT64 }))
    .OP_END_FACTORY_REG(Add);

REG_OP(Mul)
    .INPUT(x1, TensorType({ DT_INT64 }))
    .INPUT(x2, TensorType({ DT_INT64 }))
    .OUTPUT(y, TensorType({ DT_INT64 }))
    .OP_END_FACTORY_REG(Mul);

REG_OP(Maximum)
    .INPUT(x1, TensorType({ DT_INT64 }))
    .INPUT(x2, TensorType({ DT_INT64 }))
    .OUTPUT(y, TensorType({ DT_INT64 }))
    .OP_END_FACTORY_REG(Maximum);

REG_OP(Minimum)
    .INPUT(x1, TensorType({ DT_INT64 }))
    .INPUT(x2, TensorType({ DT_INT64 }))
    .OUTPUT(y, TensorType({ DT_INT64 }))
    .OP_END_FACTORY_REG(Minimum);
}
#endif
template <typename T, typename std::enable_if<(!std::is_array<T>::value), int32_t>::type = 0>
static void *CreateCompileInfo()
{
    return new T();
}
template <typename T> static void DeleteCompileInfo(void * const obj)
{
    delete reinterpret_cast<T *>(obj);
}

namespace TbeReduce {
using namespace std;

constexpr u32 UB_BLOCK_SIZE = 32;
constexpr u64 TBE_REDUCE_MAX_COUNT = INT32_MAX;
constexpr u32 kMAX_TILING_DATA_SIZE = 16UL * 1024UL;
constexpr u32 kMAX_WORKSPACE_COUNT = 16;
fe::PlatFormInfos g_platFormInfos;
std::mutex g_accessFilesMutex;
std::map<HcclReduceOp, std::string> OPERATOR_MAP = { { HCCL_REDUCE_SUM, "Add" },
                                                     { HCCL_REDUCE_PROD, "Mul" },
                                                     { HCCL_REDUCE_MAX, "Maximum" },
                                                     { HCCL_REDUCE_MIN, "Minimum" } };

std::map<HcclDataType, ge::DataType> CONVERT_DATATYPE_MAP = {
    { HCCL_DATA_TYPE_INT8, ge::DataType::DT_INT8 },     { HCCL_DATA_TYPE_INT16, ge::DataType::DT_INT16 },
    { HCCL_DATA_TYPE_INT32, ge::DataType::DT_INT32 },   { HCCL_DATA_TYPE_FP16, ge::DataType::DT_FLOAT16 },
    { HCCL_DATA_TYPE_INT64, ge::DataType::DT_INT64 },   { HCCL_DATA_TYPE_UINT64, ge::DataType::DT_UINT64 },
    { HCCL_DATA_TYPE_UINT8, ge::DataType::DT_UINT8 },   { HCCL_DATA_TYPE_UINT16, ge::DataType::DT_UINT16 },
    { HCCL_DATA_TYPE_UINT32, ge::DataType::DT_UINT32 }, { HCCL_DATA_TYPE_BFP16, ge::DataType::DT_BF16 }
};

TbeVectorReduce::TbeVectorReduce()
    : binHandle_(nullptr),
      tilingDataHostPtr_(nullptr),
      binaryDataPtr_(nullptr),
      isInit_(false),
      deviceType_(DevType::DEV_TYPE_910)
{}

TbeVectorReduce::~TbeVectorReduce()
{
    if (binHandle_ != nullptr) {
        HcclResult ret = hrtDevBinaryUnRegister(binHandle_);
        if (ret != HCCL_SUCCESS) {
            HCCL_ERROR("[Destory][TbeVectorReduce]hrt dev binary unregister failed");
        }
    }

    if (binaryDataPtr_ != nullptr) {
        delete[] binaryDataPtr_;
        binaryDataPtr_ = nullptr;
    }

    if (tilingDataHostPtr_ != nullptr) {
        delete[] tilingDataHostPtr_;
        tilingDataHostPtr_ = nullptr;
    }

    for (auto iter = tilingDataMap_.begin(); iter != tilingDataMap_.end(); iter++) {
        if (iter->second != nullptr) {
            if (hrtFree(iter->second) != HCCL_SUCCESS) {
                HCCL_WARNING("free tilingdata memory failed");
            }
            iter->second = nullptr;
        }
    }

    for (auto iter = opNameStubFuncsMap_.begin(); iter != opNameStubFuncsMap_.end(); iter++) {
        free(iter->second);
        iter->second = nullptr;
    }

    std::map<int32_t, void *>::iterator devMemIter;
    for (devMemIter = devMemMap_.begin(); devMemIter != devMemMap_.end(); devMemIter++) {
        if (devMemIter->second != nullptr) {
            if (hrtFree(devMemIter->second) != HCCL_SUCCESS) {
                HCCL_WARNING("free device memory failed");
            }
            devMemIter->second = nullptr;
        }
    }
}

HcclResult TbeVectorReduce::Init()
{
    CHK_PRT_RET(isInit_, HCCL_DEBUG("TbeVectorReduce had been initialized"), HCCL_SUCCESS);

    CHK_RET(hrtGetDeviceType(deviceType_));
    CHK_PRT_RET(hccl::Is310PDevice(), HCCL_INFO("devType[%d] does not need tbe", deviceType_), HCCL_SUCCESS);

    // 初始化op信息
    switch (deviceType_) {
        case DevType::DEV_TYPE_910:
            InitOpInfoMap910A();
            GetCoreNum();
            break;
        case DevType::DEV_TYPE_310P3:
            InitOpInfoMap310P3();
            break;
        case DevType::DEV_TYPE_910B:
        case DevType::DEV_TYPE_910_93:
            InitOpInfoMap910B();
            GetCoreNum();
            break;
        default:
            HCCL_ERROR("[TbeVectorReduce][Init]devType[%u] is not supported", deviceType_);
            return HCCL_E_NOT_SUPPORT;
    }

    // 使用内存申请
    tilingDataHostPtr_ = new (std::nothrow) char[TILING_DATA_MAX_SZIE];
    CHK_PTR_NULL(tilingDataHostPtr_);

    CHK_RET(LoadOpBinary());

    isInit_ = true;
    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::LoadOpBinary()
{
    // 读取op二进制文件
    rtDevBinary_t opBinary;
    opBinary.version = 0;

    if (deviceType_ == DevType::DEV_TYPE_910 || deviceType_ == DevType::DEV_TYPE_310P3) {
        opBinary.magic = RT_DEV_BINARY_MAGIC_ELF;
    } else if (deviceType_ == DevType::DEV_TYPE_910B || deviceType_ == DevType::DEV_TYPE_910_93) {
        opBinary.magic = RT_DEV_BINARY_MAGIC_ELF_AIVEC;
    }

    opBinary.length = 0;
    CHK_RET(LoadOpBinaryWithParams(opBinary.data, opBinary.length));

    // 注册op二进制文件到device
    HcclResult ret = hrtDevBinaryRegister(&opBinary, &binHandle_);
    CHK_PRT_RET(ret != HCCL_SUCCESS,
        HCCL_ERROR("[Init][TbeVectorReduce]register opBinary failed binaryLenth[%lld]", opBinary.length),
        HCCL_E_RUNTIME);
    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::Run(const void *src1, const void *src2, u64 count, const HcclDataType dataType,
    HcclReduceOp redOp, void *stream, const void *dst)
{
    CHK_PRT_RET(!isInit_, HCCL_ERROR("[Run][Reduce]TbeVectorReduce had not been initialized"), HCCL_E_UNAVAIL);

    CHK_SMART_PTR_NULL(src1);
    CHK_SMART_PTR_NULL(src2);
    CHK_SMART_PTR_NULL(dst);
    CHK_SMART_PTR_NULL(stream);
    CHK_PRT_RET(count == 0, HCCL_WARNING("input parameter 'count' is 0"), HCCL_SUCCESS);
    CHK_PRT_RET(dataType >= HCCL_DATA_TYPE_RESERVED, HCCL_ERROR("[Run][Reduce]unsupported data type[%s]",
        GetDataTypeEnumStr(dataType).c_str()), HCCL_E_PARA);

    // 获取总的数据量
    u64 dataSize = SIZE_TABLE[dataType] * count;                                     // 计算总的字节数
    u64 headCount = dataSize / UB_BLOCK_SIZE * UB_BLOCK_SIZE / SIZE_TABLE[dataType]; // 计算出32字节整倍数的数据数量
    u64 tailCount = count - headCount;

    if (headCount != 0) { // 对满足32字节整倍数的数据进行reduce
        CHK_RET(VectorReduceLoop(src1, src2, headCount, dataType, redOp, stream, dst));
    }
    if (tailCount != 0) { // 对不满足32字节整倍数的剩余数据进行reduce
        void *tailSrc1 = static_cast<char *>(const_cast<void *>(src1)) + (headCount * SIZE_TABLE[dataType]);
        void *tailSrc2 = static_cast<char *>(const_cast<void *>(src2)) + (headCount * SIZE_TABLE[dataType]);
        void *tailDst = static_cast<char *>(const_cast<void *>(dst)) + (headCount * SIZE_TABLE[dataType]);
        CHK_RET(TailVectorReduce(tailSrc1, tailSrc2, tailCount, dataType, redOp, stream, tailDst));
    }
    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::GetKernelFunction(const HcclDataType dataType, const HcclReduceOp redOp,
    nlohmann::json &opDescInfo, OpRunInfo &runInfo, s8 *&stubFunc)
{
    // 构造桩函数
    // 获取stub函数
    std::string opName;
    CHK_RET(GetOpProperty(opDescInfo, "binFileName", opName));

    std::string kernelName = "";
    CHK_RET(GetKernelName(opDescInfo, dataType, kernelName, runInfo, redOp));

    std::unique_lock<std::mutex> stubFuncLock(stubFuncMutex_);
    CHK_RET(GetstubFunc(opName, stubFunc, kernelName));
    stubFuncLock.unlock();

    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::GetTilingDataInfo(const u64 count, const HcclDataType dataType, const HcclReduceOp redOp,
    void *&tilingDataDevPtr, nlohmann::json &opDescInfo, OpRunInfo &runInfo)
{
    runInfo.blockDim = 0;
    runInfo.workspaces = { 0 };
    runInfo.clearAtomic = 0;
    HCCL_INFO("VectorReduce count[%llu], dataType[%s]", count, GetDataTypeEnumStr(dataType).c_str());
    if ((IsSupportOperatorDevType910A(dataType, redOp))) {
#ifndef HCCD
        CHK_RET(GetOpInfo(dataType, redOp, opDescInfo));
        // 获取tiling的执行信息
        HCCL_INFO("opDescInfo[%s]", opDescInfo.dump().c_str());
        CHK_RET(GetTilingRunInfo(opDescInfo, count, dataType, runInfo, redOp));
#else
        HCCL_ERROR("[TbeVectorReduce]Does not support this interface.");
        return HCCL_E_NOT_SUPPORT;
#endif
    } else {
        nlohmann::json opTilingInfo;
        CHK_RET(GetOpInfo(dataType, redOp, opDescInfo, opTilingInfo));
        // 获取tiling的执行信息
        HCCL_INFO("opDescInfo[%s]", opDescInfo.dump().c_str());
        CHK_RET(GetTilingRunInfo(opDescInfo, opTilingInfo, count, dataType, runInfo, redOp));
    }

    HCCL_INFO("runInfo blockDim[%u], clearAtomic[%d], tilingKey[%u]",
        runInfo.blockDim, runInfo.clearAtomic, runInfo.tilingKey);

    // 如果tilingDataMap_里没有，用GetTilingDataDevMem获取tilingDataDevPtr；如果tilingDataMap_里有，直接取出来给tilingDataDevPtr
    TilingInputInfo tilingDataInfo = { dataType, redOp, count };
    std::unique_lock<std::mutex> tilingDataLock(tilingDataMutex_);
    if (tilingDataMap_.find(tilingDataInfo) == tilingDataMap_.end()) {
        CHK_RET(GetTilingDataDevMem(runInfo, &tilingDataDevPtr, TILING_DATA_MAX_SZIE));
        tilingDataMap_[tilingDataInfo] = tilingDataDevPtr;
    } else {
        tilingDataDevPtr = tilingDataMap_[tilingDataInfo];
    }
    tilingDataLock.unlock();

    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::JudgeIsSupported(const HcclDataType dataType, const HcclReduceOp redOp)
{
    // 判断是否支持某设备类型+数据类型+reduce类型
    nlohmann::json opDescInfo;
    if (IsSupportOperatorDevType910A(dataType, redOp)) {
#ifndef HCCD
        CHK_RET(GetOpInfo(dataType, redOp, opDescInfo)); // 尝试获取opDescInfo，获取不到则表示不支持，报错
#else
        HCCL_ERROR("[TbeVectorReduce][JudgeIsSupported] Does not support this interface.");
        return HCCL_E_NOT_SUPPORT;
#endif
    } else {
        nlohmann::json opTilingInfo;
        CHK_RET(GetOpInfo(dataType, redOp, opDescInfo, opTilingInfo)); // 尝试获取opDescInfo和opTilingInfo，获取不到则表示不支持，报错
    }

    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::VectorReduce(const void *src1, const void *src2, u64 count, const HcclDataType dataType,
    HcclReduceOp redOp, void *stream, const void *dst)
{
    // 获取op关联的描述信息和tiling的执行信息
    nlohmann::json opDescInfo;
    OpRunInfo runInfo;
    void *tilingDataDevPtr = nullptr;
    CHK_RET(GetTilingDataInfo(count, dataType, redOp, tilingDataDevPtr, opDescInfo, runInfo));

    s8 *stubFunc = nullptr;
    CHK_RET(GetKernelFunction(dataType, redOp, opDescInfo, runInfo, stubFunc));

    // 执行OP
    CHK_RET(
        ExecuteKernelLaunch(src1, src2, dst, stream, reinterpret_cast<void *>(stubFunc), runInfo, tilingDataDevPtr));
    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::VectorReduceLoop(const void *src1, const void *src2, u64 count,
    const HcclDataType dataType, HcclReduceOp redOp, void *stream, const void *dst)
{
    const u32 unitSize = SIZE_TABLE[dataType];
    void *currentSrc1 = const_cast<void *>(src1);
    void *currentSrc2 = const_cast<void *>(src2);
    void *currentDst = const_cast<void *>(dst);
 
    // 计算出字节数为32字节整倍数的最大count
    const u64 maxCountPerLoop = ((TBE_REDUCE_MAX_COUNT * unitSize) / UB_BLOCK_SIZE * UB_BLOCK_SIZE) / unitSize;
    u64 countLeft = count;
 
    while (countLeft > 0) {
        u64 currentCount = countLeft > maxCountPerLoop ? maxCountPerLoop : countLeft;
        HCCL_DEBUG(
            "[VectorReduceLoop] currentCount[%llu], countLeft[%llu], currentSrc1[%p], currentSrc2[%p], currentDst[%p]",
            currentCount, countLeft, currentSrc1, currentSrc2, currentDst);
 
        CHK_RET(VectorReduce(currentSrc1, currentSrc2, currentCount, dataType, redOp, stream, currentDst));
 
        currentSrc1 = static_cast<void *>(static_cast<s8 *>(currentSrc1) + currentCount * unitSize);
        currentSrc2 = static_cast<void *>(static_cast<s8 *>(currentSrc2) + currentCount * unitSize);
        currentDst = static_cast<void *>(static_cast<s8 *>(currentDst) + currentCount * unitSize);
        countLeft -= currentCount;
    }
    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::GetstubFunc(std::string &opName, s8 *&stubFunc, std::string &kernelName)
{
    if (opNameStubFuncsMap_.find(kernelName) == opNameStubFuncsMap_.end()) {
        stubFunc = static_cast<s8 *>(malloc(opName.length() + 1));
        CHK_PTR_NULL(stubFunc);
        opNameStubFuncsMap_[kernelName] = stubFunc;
        if (memcpy_s(stubFunc, opName.length() + 1, opName.c_str(), opName.length()) != EOK) {
            HCCL_ERROR("[Reduce][Vector]stubFunc mem copy failed");
            return HCCL_E_MEMORY;
        }
        *(stubFunc + opName.length()) =  '\0';
        CHK_RET(GenerateStubFunc(stubFunc, kernelName));
    } else {
        stubFunc = opNameStubFuncsMap_[kernelName];
    }

    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::DevMemMalloc(void *stream, void *&devMem1, void *&devMem2)
{
    int32_t streamId;
    CHK_RET(hrtGetStreamId(stream, streamId));

    std::unique_lock<std::mutex> lock(devMemMutex_);
    if (devMemMap_.find(streamId) == devMemMap_.end()) {
        CHK_RET(hrtMalloc(&devMem1, UB_BLOCK_SIZE + UB_BLOCK_SIZE));
        CHK_PTR_NULL(devMem1);
        devMem2 = static_cast<char *>(devMem1) + UB_BLOCK_SIZE;
        devMemMap_[streamId] = devMem1;
    } else {
        devMem1 = devMemMap_[streamId];
        devMem2 = static_cast<char *>(devMem1) + UB_BLOCK_SIZE;
    }
    lock.unlock();

    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::TailVectorReduce(const void *tailSrc1, const void *tailSrc2, u64 tailCount,
    const HcclDataType dataType, HcclReduceOp redOp, void *stream, void *tailDst)
{
    // 申请device内存
    void *devMem1 = nullptr;
    void *devMem2 = nullptr;
    CHK_RET(DevMemMalloc(stream, devMem1, devMem2));

    // 内存拷贝
    CHK_RET(hrtMemAsyncCopy(devMem1, UB_BLOCK_SIZE, tailSrc1, tailCount * SIZE_TABLE[dataType],
        HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_DEVICE_TO_DEVICE, stream));
    CHK_RET(hrtMemAsyncCopy(devMem2, UB_BLOCK_SIZE, tailSrc2, tailCount * SIZE_TABLE[dataType],
        HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_DEVICE_TO_DEVICE, stream));

    // 算子加载
    CHK_RET(VectorReduce(devMem1, devMem2, tailCount, dataType, redOp, stream, devMem2));
    CHK_RET(hrtMemAsyncCopy(tailDst, tailCount * SIZE_TABLE[dataType], devMem2, tailCount * SIZE_TABLE[dataType],
        HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_DEVICE_TO_DEVICE, stream));

    return HCCL_SUCCESS;
}

bool TbeVectorReduce::IsSupportOperatorOverflowDevType910A(HcclDataType dataType, HcclReduceOp redOp) const
{
    bool checkDataType = (dataType == HCCL_DATA_TYPE_FP32 || dataType == HCCL_DATA_TYPE_FP16);
    bool checkDeviceType = (deviceType_ == DevType::DEV_TYPE_910);
    bool checkReduceType = (redOp == HCCL_REDUCE_SUM || redOp == HCCL_REDUCE_PROD);
    return checkDataType && checkDeviceType && checkReduceType;
}

bool TbeVectorReduce::IsSupportOperatorDevType910A(HcclDataType dataType, HcclReduceOp redOp) const
{
    bool checkDataType = (dataType == HCCL_DATA_TYPE_INT64);
    bool checkDeviceType = (deviceType_ == DevType::DEV_TYPE_910 || deviceType_ == DevType::DEV_TYPE_910B ||
        deviceType_ == DevType::DEV_TYPE_910_93);
    bool checkReduceType =
        (redOp == HCCL_REDUCE_SUM || redOp == HCCL_REDUCE_PROD || redOp == HCCL_REDUCE_MAX || redOp == HCCL_REDUCE_MIN);

    return checkDataType && checkDeviceType && checkReduceType;
}

HcclResult TbeVectorReduce::GetKernelName(nlohmann::json &opInfo, HcclDataType dataType, std::string &kernelName,
    OpRunInfo &runInfo, HcclReduceOp redOp)
{
    CHK_RET(GetOpProperty(opInfo, "kernelName", kernelName));

    if ((dataType == HCCL_DATA_TYPE_INT8) || IsSupportOperatorOverflowDevType910A(dataType, redOp) ||
        deviceType_ == DevType::DEV_TYPE_910B || deviceType_ == DevType::DEV_TYPE_910_93 ||
        IsSupportOperatorDevType910A(dataType, redOp)) {
        kernelName += '_';
        kernelName += to_string(runInfo.tilingKey);
    }
    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::LoadOpBinaryWithParams(const void *&binaryData, uint64_t &binaryLength)
{
    std::string binaryPath;
    CHK_RET(GetOpBinaryPath(binaryPath));
    // 将二进制文件读取到内存中
    std::lock_guard<std::mutex> lock(g_accessFilesMutex);
    std::ifstream infile(binaryPath, std::ios::binary);
    CHK_PRT_RET(!infile.is_open(),
        HCCL_ERROR("[Load][OpBinary]open op binary file[%s] failed,"
        "please check LD_LIBRARY_PATH and ensure that the file is exist",
        binaryPath.c_str()),
        HCCL_E_UNAVAIL);

    infile.seekg(0, infile.end);
    binaryLength = infile.tellg(); // 获取二进制文件大小
    infile.seekg(0, infile.beg);

    CHK_PRT_RET(binaryLength == 0, HCCL_ERROR("[Load][OpBinary]op binary file size is 0"), HCCL_E_PARA);
    if (binaryDataPtr_ == nullptr) {
        binaryDataPtr_ = new (std::nothrow) char[binaryLength]; // op二进制文件由类保存，在析构时销毁
        if (binaryDataPtr_ == nullptr) {
            HCCL_ERROR("[Load][OpBinary]malloc memory for binary data failed");
            infile.close();
            return HCCL_E_UNAVAIL;
        }
    }

    infile.read(binaryDataPtr_, binaryLength);
    binaryData = binaryDataPtr_;
    infile.close();

    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::GetOpBinaryPath(std::string &binaryPath)
{
    // 获取二进制文件路径
    std::string libPath;
    std::string getPath = SalGetEnv("LD_LIBRARY_PATH");
    if (getPath != "EmptyString") {
        libPath = getPath;
    } else {
        HCCL_WARNING("ENV:LD_LIBRARY_PATH is not set");
    }

    size_t mid = libPath.find("fwkacllib/lib64");
    if (mid == libPath.npos) {
        HCCL_WARNING("ENV:LD_LIBRARY_PATH lack fwkacllib/lib64");

        mmDlInfo info;
        mmDladdr(reinterpret_cast<void *>(HcclDlopen), &info);

        CHK_PRT_RET(info.dli_fname == nullptr, HCCL_ERROR("[Get][OpBinaryPath]get path of libhccl_plf.so failed"),
            HCCL_E_UNAVAIL);

        char resolvedPath[PATH_MAX];
        if (realpath(info.dli_fname, resolvedPath) == nullptr) {
            HCCL_ERROR("[Get][OpBinaryPath]path %s is not a valid real path", info.dli_fname);
            return HCCL_E_INTERNAL;
        }
        binaryPath = resolvedPath;
        if (binaryPath.find("/libhccl_plf.so") != binaryPath.npos) {
            binaryPath.erase(binaryPath.find("/libhccl_plf.so"));
        } else {
            HCCL_ERROR("[Get][OpBinaryPath]get binary path failed");
            return HCCL_E_PARA;
        }
        HCCL_DEBUG("op binary file path[%s]", binaryPath.c_str());
    } else {
        u32 diff;
        if (libPath.find(":", mid) == libPath.npos) {
            diff = libPath.length() - libPath.rfind(":", mid);
        } else {
            diff = libPath.find(":", mid) - libPath.rfind(":", mid);
        }
        binaryPath = libPath.substr(libPath.rfind(":", mid) + 1, diff - 1);
    }

    // 判断应该加载的文件
    switch (deviceType_) {
        default:
        case DevType::DEV_TYPE_910:
            binaryPath += "/hccl_reduce_op_ascend910.o";
            break;
        case DevType::DEV_TYPE_910B:
        case DevType::DEV_TYPE_910_93:
            binaryPath += "/hccl_reduce_op_ascend910B.o";
            break;
        case DevType::DEV_TYPE_310P3:
            binaryPath += "/hccl_reduce_op_Ascend310P3.o";
            break;
        case DevType::DEV_TYPE_310P1:
            HCCL_ERROR("[Get][OpBinaryPath]devType[%u] is not supported", deviceType_);
            return HCCL_E_NOT_SUPPORT;
    }

    return HCCL_SUCCESS;
}

template <typename T, bool IsPointer = std::is_pointer<T>::value>
void PrintHex(const T *p, size_t num, std::stringstream &ss)
{
    for (size_t i = 0; i < num; ++i) {
        if (!IsPointer) {
            // 通过std::setw设置输出位宽为2倍的sizeof(T)
            ss << "0x" << std::setfill('0') << std::setw(static_cast<int32_t>(sizeof(T)) * 2) << std::hex << +p[i] <<
                ' ';
        } else {
            ss << p[i] << ' ';
        }
    }
}

HcclResult TbeVectorReduce::GetTilingDataDevMem(OpRunInfo &runInfo, void **tilingDataDevPtr,
    const u32 tilingDataMaxSize)
{
    // 处理tiling data
    runInfo.tilingData.seekg(0, std::ios::end);
    u64 tilingDataLength = runInfo.tilingData.tellg();
    runInfo.tilingData.seekg(0, std::ios::beg);

    CHK_PRT_RET(tilingDataLength > tilingDataMaxSize,
        HCCL_ERROR("[Get][TilingData]tiling data length[%llu] > max size[%u]", tilingDataLength, tilingDataMaxSize),
        HCCL_E_INTERNAL);

    {
        runInfo.tilingData.read(tilingDataHostPtr_, tilingDataLength);
        std::stringstream ss;
        PrintHex(reinterpret_cast<u8 *>(tilingDataHostPtr_), tilingDataLength, ss);
        HCCL_INFO("[GetTilingDataDevMem]tilingData[%s]", ss.str().c_str());

        CHK_RET(hrtMalloc(tilingDataDevPtr, tilingDataLength));
        HcclResult ret = hrtMemcpy(*tilingDataDevPtr, tilingDataLength, tilingDataHostPtr_, tilingDataLength,
            HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE);
        CHK_PRT_RET(ret != HCCL_SUCCESS,
            HCCL_ERROR("[Get][TilingData]tiling data memcpy host to device failed,"
            "size[%llu]",
            tilingDataLength),
            HCCL_E_RUNTIME);
    }
    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::SetGlobalWorkSpace(std::vector<void *> &globalWorkSpaceAddr)
{
    if (globalWorkSpaceAddr.size() != 0) {
        HCCL_INFO("enable tbe_vector_reduce overflow detection");
        globalWorkSpaceAddr_.assign(globalWorkSpaceAddr.begin(), globalWorkSpaceAddr.end());
    }
    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::ConcatMem(const void *src1, const void *src2, const void *dst, const void *tilingDataDevPtr,
    std::vector<const void *> &deviceAddrs)
{
    deviceAddrs.push_back(src2);
    deviceAddrs.push_back(src1);
    deviceAddrs.push_back(dst);
    deviceAddrs.push_back(tilingDataDevPtr);

    // 导入代表溢出检测的内存
    if (globalWorkSpaceAddr_.size() != 0) {
        for (auto &dumpAttr : globalWorkSpaceAddr_) {
            deviceAddrs.push_back(dumpAttr);
        }
    }
    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::ExecuteKernelLaunch(const void *src1, const void *src2, const void *dst, void *stream,
    const void *stubFunc, OpRunInfo &runInfo, void *tilingDataDevPtr)
{
    std::vector<const void *> deviceAddrs;
    CHK_RET(ConcatMem(src1, src2, dst, tilingDataDevPtr, deviceAddrs));

    HcclResult ret = hrtKernelLaunch(stubFunc, runInfo.blockDim, deviceAddrs.data(),
        deviceAddrs.size() * sizeof(void *), nullptr, stream);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Execute][KernelLaunch]execute kernel launch failed"), HCCL_E_RUNTIME);
    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::GetTilingRunInfo(nlohmann::json &opDescInfo, nlohmann::json &opTilingInfo, u64 count,
    HcclDataType dataType, OpRunInfo &runInfo, HcclReduceOp redOp)
{
    // 构造tiling入参，进行tiling操作
    TeOpTensor inputTensor;
    switch (dataType) {
        case HCCL_DATA_TYPE_INT32:
            inputTensor.dtype = "int32";
            break;
        case HCCL_DATA_TYPE_FP32:
            inputTensor.dtype = "float32";
            break;
        case HCCL_DATA_TYPE_FP16:
            inputTensor.dtype = "float16";
            break;
        case HCCL_DATA_TYPE_INT8:
            inputTensor.dtype = "int8";
            break;
        case HCCL_DATA_TYPE_INT16:
            inputTensor.dtype = "int16";
            break;
        default:
            CHK_PRT_RET(true, HCCL_ERROR("[Get][TilingRunInfo]unsupported data type[%s]",
                GetDataTypeEnumStr(dataType).c_str()), HCCL_E_PARA);
            break;
    }

    inputTensor.shape = { 1, static_cast<s64>(count) };
    inputTensor.oriShape = { 1, static_cast<s64>(count) };
    inputTensor.format = "ND";
    inputTensor.oriFormat = "ND";

    TeOpTensorArg tilingArg;
    tilingArg.tensor.push_back(inputTensor);
    tilingArg.argType = TensorArgType::TA_SINGLE;

    TeOpParas paras;
    paras.inputs.push_back(tilingArg);
    paras.inputs.push_back(tilingArg);
    paras.outputs.push_back(tilingArg);

    std::string opName;
    CHK_RET(GetOpProperty(opDescInfo, "binFileName", opName));

    if (IsSupportOperatorOverflowDevType910A(dataType, redOp) || deviceType_ == DevType::DEV_TYPE_910B ||
        deviceType_ == DevType::DEV_TYPE_910_93) {
        bool isTiling = EletwiseTilingV3(opName, paras, opTilingInfo, runInfo);
        CHK_PRT_RET(!isTiling, HCCL_ERROR("[Get][TilingRunInfo]tbe op tiling failed"), HCCL_E_CCE);
    } else if (dataType != HCCL_DATA_TYPE_INT8) {
        bool isTiling = EletwiseTilingV1(opName, paras, opTilingInfo, runInfo);
        CHK_PRT_RET(!isTiling, HCCL_ERROR("[Get][TilingRunInfo]tbe op tiling failed"), HCCL_E_CCE);
    } else {
        bool isTiling = EletwiseTilingV2(opName, paras, opTilingInfo, runInfo);
        CHK_PRT_RET(!isTiling, HCCL_ERROR("[Get][TilingRunInfo]tbe op tiling failed"), HCCL_E_CCE);
    }

    return HCCL_SUCCESS;
}
#ifndef HCCD

HcclResult TbeVectorReduce::GetOpInfo(HcclDataType dataType, HcclReduceOp redOp, nlohmann::json &opDescInfo)
{
    std::string opInfoIndex;
    CHK_RET(GetOpInfoIndex(opInfoIndex, dataType, redOp));

    if (opInfoMap_.find(opInfoIndex) == opInfoMap_.end()) {
        HCCL_ERROR("[Get][OpInfo]can't find tbereduce info,opInfoIndex[%s]", opInfoIndex.c_str());
        return HCCL_E_UNAVAIL;
    }
    opDescInfo = opInfoMap_[opInfoIndex];
    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::GetTilingRunInfo(nlohmann::json &opDescInfo, u64 count, HcclDataType dataType,
    OpRunInfo &runInfo, HcclReduceOp redOp)
{
    if (CONVERT_DATATYPE_MAP.count(dataType) == 0) {
        HCCL_ERROR("[Get][dataType]data type[%s] is not supported", GetDataTypeEnumStr(dataType).c_str());
        return HCCL_E_PARA;
    }
    ge::OpDescPtr op_desc = nullptr;
    EXECEPTION_CATCH(op_desc = make_shared<ge::OpDesc>("tmp_op", OPERATOR_MAP[redOp].c_str()), return HCCL_E_PTR);
    std::vector<int64_t> dims = { 1, static_cast<s64>(count) };
    ge::GeTensorDesc tensor_desc(ge::GeShape(dims), ge::FORMAT_ND, CONVERT_DATATYPE_MAP[dataType]);
    ge::GeTensorDesc tensor_desc1(ge::GeShape(dims), ge::FORMAT_ND, CONVERT_DATATYPE_MAP[dataType]);

    op_desc->AddInputDesc("x1", tensor_desc);
    op_desc->AddInputDesc("x2", tensor_desc1);
    op_desc->AddOutputDesc("y", tensor_desc1);
    ge::ComputeGraphPtr graph = nullptr;
    EXECEPTION_CATCH(graph = make_shared<ge::ComputeGraph>("tmp"), return HCCL_E_PTR);
    ge::NodePtr node = graph->AddNode(op_desc);
    const size_t INPUT_SIZE = 2;
    CHK_RET(AddDataNodeForOriginNode(graph, node, INPUT_SIZE));
    ge::Operator op = ge::OpDescUtils::CreateOperatorFromNode(node);

    // 解析
    std::string compileInfoJson = opDescInfo.at("compileInfo").dump();
    HCCL_INFO("compileInfoJson[%s]", compileInfoJson.c_str());

    auto tilingParseContext =
        gert::TilingParseContextBuilder()
            .CompileJson(compileInfoJson.c_str())
            .PlatformInfo(reinterpret_cast<void *>(const_cast<fe::PlatFormInfos *>(&g_platFormInfos)))
            .CompileInfoCreatorFunc(CreateCompileInfo<AutoTilingCompileInfo>)
            .CompileInfoDeleterFunc(DeleteCompileInfo<AutoTilingCompileInfo>)
            .Build(op);

    auto parseContext = reinterpret_cast<gert::TilingParseContext *>(tilingParseContext.GetKernelContext());
    CHK_PTR_NULL(parseContext);

    ge::graphStatus gRet = AutoTilingParser(parseContext);
    CHK_PRT_RET(gRet != ge::GRAPH_SUCCESS, HCCL_ERROR("[GetTilingRunInfo] Auto TilingParser failed"), HCCL_E_PARA);

    auto compiledInfo = parseContext->GetCompiledInfo<AutoTilingCompileInfo>();

    // 构造tiling入参，进行tiling操作
    uint32_t maxSize = -1;
    if (GetOpProperty(opDescInfo, "opParaSize", maxSize) != HCCL_SUCCESS) {
        HCCL_INFO("No max tiling size in opdesc.");
        maxSize = kMAX_TILING_DATA_SIZE;
    }
    const auto alignedMaxSize = RoundUp(static_cast<uint64_t>(maxSize), sizeof(uintptr_t));
    const auto tilingData = gert::TilingData::CreateCap(alignedMaxSize);
    const auto workspaceSize = gert::ContinuousVector::Create<size_t>(kMAX_WORKSPACE_COUNT);
    auto context_builder = gert::TilingContextBuilder();
    const gert::KernelContextHolder tilingContext =
        context_builder
            .CompileInfo(reinterpret_cast<void *>(compiledInfo))
            .PlatformInfo(reinterpret_cast<void *>(const_cast<fe::PlatFormInfos *>(&g_platFormInfos)))
            .TilingData(tilingData.get())
            .Workspace(reinterpret_cast<gert::ContinuousVector *>(workspaceSize.get()))
            .Build(op);
    gert::TilingContext *context = reinterpret_cast<gert::TilingContext *>(tilingContext.context_);
    CHK_PTR_NULL(context);

    auto rawTilingData = context->GetRawTilingData();
    if (rawTilingData != nullptr) {
        rawTilingData->SetDataSize(0);
    }

    gRet = AutoTiling(context);
    CHK_PRT_RET(gRet != ge::GRAPH_SUCCESS, HCCL_ERROR("[GetTilingRunInfo] Auto Tiling failed"), HCCL_E_PARA);
    CHK_RET(ConvertToOpRunInfo(context, runInfo));
    return HCCL_SUCCESS;
}
#endif
HcclResult TbeVectorReduce::GenerateStubFunc(s8 *&stubFunc, string &kernelName)
{
    HCCL_DEBUG("[GenerateStubFunc]stubFunc[%s], kernelName[%s]", stubFunc, kernelName.c_str());
    const char *stubName = reinterpret_cast<char *>(stubFunc);
    HcclResult ret = hrtFunctionRegister(binHandle_, stubFunc, stubName, kernelName.c_str(), 0);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Generate][StubFunc]function register failed"), HCCL_E_PARA);
    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::GetOpInfo(HcclDataType dataType, HcclReduceOp redOp, nlohmann::json &opDescInfo,
    nlohmann::json &opTilingInfo)
{
    std::string opInfoIndex;
    CHK_RET(GetOpInfoIndex(opInfoIndex, dataType, redOp));

    if (opInfoMap_.find(opInfoIndex) == opInfoMap_.end()) {
        HCCL_ERROR("[Get][OpInfo]can't find tbereduce info,opInfoIndex[%s]", opInfoIndex.c_str());
        return HCCL_E_UNAVAIL;
    }
    opDescInfo = opInfoMap_[opInfoIndex];

    opInfoIndex += "_tiling_info";
    if (opInfoMap_.find(opInfoIndex) == opInfoMap_.end()) {
        HCCL_ERROR("[Get][OpInfo]can't find tbereduce info,opInfoTilingIndex[%s]", opInfoIndex.c_str());
        return HCCL_E_UNAVAIL;
    }
    opTilingInfo = opInfoMap_[opInfoIndex];

    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::GetOpInfoIndex(std::string &opInfoIndex, HcclReduceOp redOp)
{
    switch (redOp) {
        case HCCL_REDUCE_SUM:
            opInfoIndex += "dynamic_add_";
            break;
        case HCCL_REDUCE_PROD:
            opInfoIndex += "dynamic_mul_";
            break;
        case HCCL_REDUCE_MAX:
            opInfoIndex += "dynamic_maximum_";
            break;
        case HCCL_REDUCE_MIN:
            opInfoIndex += "dynamic_minimum_";
            break;
        default:
            CHK_PRT_RET(true, HCCL_ERROR("[Get][OpInfoIndex]calculation type is not supported"), HCCL_E_PARA);
            break;
    }

    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::GetOpInfoIndex(std::string &opInfoIndex, HcclDataType dataType)
{
    switch (dataType) {
        case HCCL_DATA_TYPE_INT32:
            opInfoIndex += "int32_";
            break;
        case HCCL_DATA_TYPE_FP32:
            opInfoIndex += "float32_";
            break;
        case HCCL_DATA_TYPE_FP16:
            opInfoIndex += "float16_";
            break;
        case HCCL_DATA_TYPE_INT8:
            opInfoIndex += "int8_";
            break;
        case HCCL_DATA_TYPE_INT16:
            opInfoIndex += "int16_";
            break;
        case HCCL_DATA_TYPE_INT64:
            opInfoIndex += "int64_";
            break;
        default:
            RPT_INPUT_ERR(true, "EI0003", std::vector<std::string>({ "ccl_op", "parameter", "value", "tips" }),
                std::vector<std::string>({
                    "GetOpInfoIndex", "dataType", std::to_string(dataType), "please check dataType"
                }));
            CHK_PRT_RET(true, HCCL_ERROR("[Get][OpInfoIndex]data type is not supported"), HCCL_E_PARA);
            break;
    }

    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::GetOpInfoIndex(std::string &opInfoIndex)
{
    switch (deviceType_) {
        default:
        case DevType::DEV_TYPE_910:
            opInfoIndex += "v80";
            break;
        case DevType::DEV_TYPE_910B:
        case DevType::DEV_TYPE_910_93:
            opInfoIndex += "v81";
            break;
        case DevType::DEV_TYPE_310P3:
            opInfoIndex += "v51";
            break;
        case DevType::DEV_TYPE_310P1:
            HCCL_ERROR("[Get][OpBinaryPath]devType[%u] is not supported", deviceType_);
            return HCCL_E_NOT_SUPPORT;
    }

    if (deviceType_ == DevType::DEV_TYPE_910B || deviceType_ == DevType::DEV_TYPE_910_93) {
        s8 chipVer[CHIP_VERSION_MAX_LEN] = {0};
        CHK_RET(hrtGetSocVer(chipVer, CHIP_VERSION_MAX_LEN));
        auto iter = OPINFO_INDEX_SOC_STRING_MAP.find(reinterpret_cast<char *>(chipVer));
        CHK_PRT_RET(iter == OPINFO_INDEX_SOC_STRING_MAP.end(),
                    HCCL_ERROR("[Get][OpInfoIndex]chipVer[%s] is not supported",
                    chipVer), HCCL_E_NOT_SUPPORT);
        opInfoIndex += iter->second;
    }

    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::GetOpInfoIndex(std::string &opInfoIndex, HcclDataType dataType, HcclReduceOp redOp)
{
    // 规约类型适配
    CHK_RET(GetOpInfoIndex(opInfoIndex, redOp));
    // 数据类型适配
    CHK_RET(GetOpInfoIndex(opInfoIndex, dataType));
    // 芯片类型适配
    CHK_RET(GetOpInfoIndex(opInfoIndex));

    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::GetOpProperty(nlohmann::json &opInfo, const std::string propName, std::string &propValue)
{
    /* 查找json对象中是否有该属性, 不存在的属性不能直接访问 */
    if (opInfo.find(propName) == opInfo.end()) {
        HCCL_ERROR("[Get][OpProperty]property does not exist");
        return HCCL_E_PARA;
    }
    /* 判断是否是字符串 */
    if (opInfo[propName].is_string()) {
        propValue = static_cast<std::string>(opInfo[propName]);
        return HCCL_SUCCESS;
    } else {
        HCCL_ERROR("[Get][OpProperty]data type is not supported");
        return HCCL_E_PARA;
    }
}

HcclResult TbeVectorReduce::GetOpProperty(nlohmann::json &opInfo, const std::string propName, u32 &propValue)
{
    /* 查找json对象中是否有该属性, 不存在的属性不能直接访问 */
    if (opInfo.find(propName) == opInfo.end()) {
        HCCL_ERROR("[Get][OpProperty]property does not exist");
        return HCCL_E_PARA;
    }
    /* 判断是否是数字 */
    if (opInfo[propName].is_number()) {
        propValue = static_cast<u32>(opInfo[propName]);
        return HCCL_SUCCESS;
    } else {
        HCCL_ERROR("[Get][OpProperty]data type is not supported");
        return HCCL_E_PARA;
    }
}
#ifndef HCCD
HcclResult TbeVectorReduce::AddDataNodeForOriginNode(ge::ComputeGraphPtr &graph, ge::NodePtr &origin_node,
    size_t input_size)
{
    // add data node for output
    for (size_t i = 0U; i < input_size; ++i) {
        ge::OpDescPtr data_op_desc = nullptr;
        EXECEPTION_CATCH(data_op_desc = std::make_shared<ge::OpDesc>(origin_node->GetName() + "_Data_" + std::to_string(i), "Data"), return HCCL_E_PTR);
        if (data_op_desc->AddOutputDesc(ge::GeTensorDesc()) != ge::SUCCESS) {
            HCCL_ERROR("data_op_desc add output desc failed, i = %zu", i);
            return HCCL_E_PARA;
        }
        auto data_node = graph->AddNode(data_op_desc);
        CHK_SMART_PTR_NULL(data_node);
        ge::graphStatus gRet = ge::GraphUtils::AddEdge(data_node->GetOutDataAnchor(0), origin_node->GetInDataAnchor(i));
        if (gRet != ge::SUCCESS) {
            HCCL_ERROR("add edge between [%s] and [%s] failed", data_node->GetName().c_str(),
                origin_node->GetName().c_str());
            return HCCL_E_PARA;
        }
    }
    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::ConvertToOpRunInfo(gert::TilingContext *context, OpRunInfo &runInfo)
{
    runInfo.blockDim = context->GetBlockDim();
    runInfo.workspaces.push_back(reinterpret_cast<int64_t>(context->GetWorkspaceSizes(context->GetWorkspaceNum())));

    auto rawTilingData = context->GetRawTilingData();
    CHK_PTR_NULL(rawTilingData);

    std::string stringTilingData(reinterpret_cast<char *>(rawTilingData->GetData()), rawTilingData->GetDataSize());
    runInfo.tilingData.str(stringTilingData);
    runInfo.clearAtomic = context->NeedAtomic();
    runInfo.tilingKey = static_cast<uint32_t>(context->GetTilingKey());

    return HCCL_SUCCESS;
}
#endif

HcclResult TbeVectorReduce::GetCoreNum()
{
#ifndef HCCD
    s32 deviceLogicId = -1;
    s64 coreNum = -1;
    HcclRtDeviceModuleType hcclModuleType = HcclRtDeviceModuleType::HCCL_RT_DEVICE_MOUDLE_RESERVED;
    CHK_RET(hrtGetDevice(&deviceLogicId));

    if (deviceType_ == DevType::DEV_TYPE_910) {
        hcclModuleType = HcclRtDeviceModuleType::HCCL_RT_MODULE_TYPE_AICORE;
    } else if (deviceType_ == DevType::DEV_TYPE_910B || deviceType_ == DevType::DEV_TYPE_910_93) {
        hcclModuleType = HcclRtDeviceModuleType::HCCL_RT_MODULE_TYPE_VECTOR_CORE;
    } else {
        HCCL_ERROR("devType[%d] does not need tbe.", deviceType_);
        return HCCL_E_INTERNAL;
    }

    CHK_RET(hrtGetDeviceInfo(deviceLogicId, hcclModuleType, HcclRtDeviceInfoType::HCCL_INFO_TYPE_CORE_NUM, coreNum));

    g_platFormInfos.SetCoreNum(static_cast<uint32_t>(coreNum));
#endif
    return HCCL_SUCCESS;
}

void TbeVectorReduce::InitOpInfoMap310P3()
{
    opInfoMap_["dynamic_add_float16_v51"] = dynamic_add_float16_v51;
    opInfoMap_["dynamic_add_float16_v51_tiling_info"] = dynamic_add_float16_v51_tiling_info;
    opInfoMap_["dynamic_add_float32_v51"] = dynamic_add_float32_v51;
    opInfoMap_["dynamic_add_float32_v51_tiling_info"] = dynamic_add_float32_v51_tiling_info;
    opInfoMap_["dynamic_add_int32_v51"] = dynamic_add_int32_v51;
    opInfoMap_["dynamic_add_int32_v51_tiling_info"] = dynamic_add_int32_v51_tiling_info;

    opInfoMap_["dynamic_mul_float16_v51"] = dynamic_mul_float16_v51;
    opInfoMap_["dynamic_mul_float16_v51_tiling_info"] = dynamic_mul_float16_v51_tiling_info;
    opInfoMap_["dynamic_mul_float32_v51"] = dynamic_mul_float32_v51;
    opInfoMap_["dynamic_mul_float32_v51_tiling_info"] = dynamic_mul_float32_v51_tiling_info;
    opInfoMap_["dynamic_mul_int32_v51"] = dynamic_mul_int32_v51;
    opInfoMap_["dynamic_mul_int32_v51_tiling_info"] = dynamic_mul_int32_v51_tiling_info;

    opInfoMap_["dynamic_maximum_float16_v51"] = dynamic_maximum_float16_v51;
    opInfoMap_["dynamic_maximum_float16_v51_tiling_info"] = dynamic_maximum_float16_v51_tiling_info;
    opInfoMap_["dynamic_maximum_float32_v51"] = dynamic_maximum_float32_v51;
    opInfoMap_["dynamic_maximum_float32_v51_tiling_info"] = dynamic_maximum_float32_v51_tiling_info;
    opInfoMap_["dynamic_maximum_int32_v51"] = dynamic_maximum_int32_v51;
    opInfoMap_["dynamic_maximum_int32_v51_tiling_info"] = dynamic_maximum_int32_v51_tiling_info;

    opInfoMap_["dynamic_minimum_float16_v51"] = dynamic_minimum_float16_v51;
    opInfoMap_["dynamic_minimum_float16_v51_tiling_info"] = dynamic_minimum_float16_v51_tiling_info;
    opInfoMap_["dynamic_minimum_float32_v51"] = dynamic_minimum_float32_v51;
    opInfoMap_["dynamic_minimum_float32_v51_tiling_info"] = dynamic_minimum_float32_v51_tiling_info;
    opInfoMap_["dynamic_minimum_int32_v51"] = dynamic_minimum_int32_v51;
    opInfoMap_["dynamic_minimum_int32_v51_tiling_info"] = dynamic_minimum_int32_v51_tiling_info;

    opInfoMap_["dynamic_add_int8_v51"] = dynamic_add_int8_v51;
    opInfoMap_["dynamic_add_int8_v51_tiling_info"] = dynamic_add_int8_v51_tiling_info;
    opInfoMap_["dynamic_mul_int8_v51"] = dynamic_mul_int8_v51;
    opInfoMap_["dynamic_mul_int8_v51_tiling_info"] = dynamic_mul_int8_v51_tiling_info;
    opInfoMap_["dynamic_maximum_int8_v51"] = dynamic_maximum_int8_v51;
    opInfoMap_["dynamic_maximum_int8_v51_tiling_info"] = dynamic_maximum_int8_v51_tiling_info;
    opInfoMap_["dynamic_minimum_int8_v51"] = dynamic_minimum_int8_v51;
    opInfoMap_["dynamic_minimum_int8_v51_tiling_info"] = dynamic_minimum_int8_v51_tiling_info;
}

void TbeVectorReduce::InitOpInfoMap910A()
{
    opInfoMap_["dynamic_add_float16_v80"] = dynamic_add_float16_v80;
    opInfoMap_["dynamic_add_float16_v80_tiling_info"] = dynamic_add_float16_v80_tiling_info;
    opInfoMap_["dynamic_add_float32_v80"] = dynamic_add_float32_v80;
    opInfoMap_["dynamic_add_float32_v80_tiling_info"] = dynamic_add_float32_v80_tiling_info;
    opInfoMap_["dynamic_add_int32_v80"] = dynamic_add_int32_v80;
    opInfoMap_["dynamic_add_int32_v80_tiling_info"] = dynamic_add_int32_v80_tiling_info;

    opInfoMap_["dynamic_mul_float16_v80"] = dynamic_mul_float16_v80;
    opInfoMap_["dynamic_mul_float16_v80_tiling_info"] = dynamic_mul_float16_v80_tiling_info;
    opInfoMap_["dynamic_mul_float32_v80"] = dynamic_mul_float32_v80;
    opInfoMap_["dynamic_mul_float32_v80_tiling_info"] = dynamic_mul_float32_v80_tiling_info;
    opInfoMap_["dynamic_mul_int32_v80"] = dynamic_mul_int32_v80;
    opInfoMap_["dynamic_mul_int32_v80_tiling_info"] = dynamic_mul_int32_v80_tiling_info;

    opInfoMap_["dynamic_maximum_float16_v80"] = dynamic_maximum_float16_v80;
    opInfoMap_["dynamic_maximum_float16_v80_tiling_info"] = dynamic_maximum_float16_v80_tiling_info;
    opInfoMap_["dynamic_maximum_float32_v80"] = dynamic_maximum_float32_v80;
    opInfoMap_["dynamic_maximum_float32_v80_tiling_info"] = dynamic_maximum_float32_v80_tiling_info;
    opInfoMap_["dynamic_maximum_int32_v80"] = dynamic_maximum_int32_v80;
    opInfoMap_["dynamic_maximum_int32_v80_tiling_info"] = dynamic_maximum_int32_v80_tiling_info;

    opInfoMap_["dynamic_minimum_float16_v80"] = dynamic_minimum_float16_v80;
    opInfoMap_["dynamic_minimum_float16_v80_tiling_info"] = dynamic_minimum_float16_v80_tiling_info;
    opInfoMap_["dynamic_minimum_float32_v80"] = dynamic_minimum_float32_v80;
    opInfoMap_["dynamic_minimum_float32_v80_tiling_info"] = dynamic_minimum_float32_v80_tiling_info;
    opInfoMap_["dynamic_minimum_int32_v80"] = dynamic_minimum_int32_v80;
    opInfoMap_["dynamic_minimum_int32_v80_tiling_info"] = dynamic_minimum_int32_v80_tiling_info;

    opInfoMap_["dynamic_add_int8_v80"] = dynamic_add_int8_v80;
    opInfoMap_["dynamic_add_int8_v80_tiling_info"] = dynamic_add_int8_v80_tiling_info;
    opInfoMap_["dynamic_mul_int8_v80"] = dynamic_mul_int8_v80;
    opInfoMap_["dynamic_mul_int8_v80_tiling_info"] = dynamic_mul_int8_v80_tiling_info;
    opInfoMap_["dynamic_maximum_int8_v80"] = dynamic_maximum_int8_v80;
    opInfoMap_["dynamic_maximum_int8_v80_tiling_info"] = dynamic_maximum_int8_v80_tiling_info;
    opInfoMap_["dynamic_minimum_int8_v80"] = dynamic_minimum_int8_v80;
    opInfoMap_["dynamic_minimum_int8_v80_tiling_info"] = dynamic_minimum_int8_v80_tiling_info;

    opInfoMap_["dynamic_add_int64_v80"] = dynamic_add_int64_v80;
    opInfoMap_["dynamic_mul_int64_v80"] = dynamic_mul_int64_v80;
    opInfoMap_["dynamic_maximum_int64_v80"] = dynamic_maximum_int64_v80;
    opInfoMap_["dynamic_minimum_int64_v80"] = dynamic_minimum_int64_v80;
}

void TbeVectorReduce::InitOpInfoMap910B()
{
    opInfoMap_["dynamic_mul_int8_v81_910B1"] = dynamic_mul_int8_v81_910B1;
    opInfoMap_["dynamic_mul_int8_v81_910B1_tiling_info"] = dynamic_mul_int8_v81_910B1_tiling_info;
    opInfoMap_["dynamic_mul_int32_v81_910B1"] = dynamic_mul_int32_v81_910B1;
    opInfoMap_["dynamic_mul_int32_v81_910B1_tiling_info"] = dynamic_mul_int32_v81_910B1_tiling_info;
    opInfoMap_["dynamic_mul_float16_v81_910B1"] = dynamic_mul_float16_v81_910B1;
    opInfoMap_["dynamic_mul_float16_v81_910B1_tiling_info"] = dynamic_mul_float16_v81_910B1_tiling_info;
    opInfoMap_["dynamic_mul_float32_v81_910B1"] = dynamic_mul_float32_v81_910B1;
    opInfoMap_["dynamic_mul_float32_v81_910B1_tiling_info"] = dynamic_mul_float32_v81_910B1_tiling_info;
    opInfoMap_["dynamic_mul_int8_v81_910B2"] = dynamic_mul_int8_v81_910B2;
    opInfoMap_["dynamic_mul_int8_v81_910B2_tiling_info"] = dynamic_mul_int8_v81_910B2_tiling_info;
    opInfoMap_["dynamic_mul_int32_v81_910B2"] = dynamic_mul_int32_v81_910B2;
    opInfoMap_["dynamic_mul_int32_v81_910B2_tiling_info"] = dynamic_mul_int32_v81_910B2_tiling_info;
    opInfoMap_["dynamic_mul_float16_v81_910B2"] = dynamic_mul_float16_v81_910B2;
    opInfoMap_["dynamic_mul_float16_v81_910B2_tiling_info"] = dynamic_mul_float16_v81_910B2_tiling_info;
    opInfoMap_["dynamic_mul_float32_v81_910B2"] = dynamic_mul_float32_v81_910B2;
    opInfoMap_["dynamic_mul_float32_v81_910B2_tiling_info"] = dynamic_mul_float32_v81_910B2_tiling_info;
    opInfoMap_["dynamic_mul_int8_v81_910B3"] = dynamic_mul_int8_v81_910B3;
    opInfoMap_["dynamic_mul_int8_v81_910B3_tiling_info"] = dynamic_mul_int8_v81_910B3_tiling_info;
    opInfoMap_["dynamic_mul_int32_v81_910B3"] = dynamic_mul_int32_v81_910B3;
    opInfoMap_["dynamic_mul_int32_v81_910B3_tiling_info"] = dynamic_mul_int32_v81_910B3_tiling_info;
    opInfoMap_["dynamic_mul_float16_v81_910B3"] = dynamic_mul_float16_v81_910B3;
    opInfoMap_["dynamic_mul_float16_v81_910B3_tiling_info"] = dynamic_mul_float16_v81_910B3_tiling_info;
    opInfoMap_["dynamic_mul_float32_v81_910B3"] = dynamic_mul_float32_v81_910B3;
    opInfoMap_["dynamic_mul_float32_v81_910B3_tiling_info"] = dynamic_mul_float32_v81_910B3_tiling_info;
    opInfoMap_["dynamic_mul_int8_v81_910B4"] = dynamic_mul_int8_v81_910B4;
    opInfoMap_["dynamic_mul_int8_v81_910B4_tiling_info"] = dynamic_mul_int8_v81_910B4_tiling_info;
    opInfoMap_["dynamic_mul_int32_v81_910B4"] = dynamic_mul_int32_v81_910B4;
    opInfoMap_["dynamic_mul_int32_v81_910B4_tiling_info"] = dynamic_mul_int32_v81_910B4_tiling_info;
    opInfoMap_["dynamic_mul_float16_v81_910B4"] = dynamic_mul_float16_v81_910B4;
    opInfoMap_["dynamic_mul_float16_v81_910B4_tiling_info"] = dynamic_mul_float16_v81_910B4_tiling_info;
    opInfoMap_["dynamic_mul_float32_v81_910B4"] = dynamic_mul_float32_v81_910B4;
    opInfoMap_["dynamic_mul_float32_v81_910B4_tiling_info"] = dynamic_mul_float32_v81_910B4_tiling_info;

    opInfoMap_["dynamic_mul_int8_v81_910B4-1"] = dynamic_mul_int8_v81_910B4;
    opInfoMap_["dynamic_mul_int8_v81_910B4-1_tiling_info"] = dynamic_mul_int8_v81_910B4_tiling_info;
    opInfoMap_["dynamic_mul_int32_v81_910B4-1"] = dynamic_mul_int32_v81_910B4;
    opInfoMap_["dynamic_mul_int32_v81_910B4-1_tiling_info"] = dynamic_mul_int32_v81_910B4_tiling_info;
    opInfoMap_["dynamic_mul_float16_v81_910B4-1"] = dynamic_mul_float16_v81_910B4;
    opInfoMap_["dynamic_mul_float16_v81_910B4-1_tiling_info"] = dynamic_mul_float16_v81_910B4_tiling_info;
    opInfoMap_["dynamic_mul_float32_v81_910B4-1"] = dynamic_mul_float32_v81_910B4;
    opInfoMap_["dynamic_mul_float32_v81_910B4-1_tiling_info"] = dynamic_mul_float32_v81_910B4_tiling_info;

    opInfoMap_["dynamic_add_int64_v81"] = dynamic_add_int64_v81;
    opInfoMap_["dynamic_mul_int64_v81"] = dynamic_mul_int64_v81;
    opInfoMap_["dynamic_maximum_int64_v81"] = dynamic_maximum_int64_v81;
    opInfoMap_["dynamic_minimum_int64_v81"] = dynamic_minimum_int64_v81;

    opInfoMap_["dynamic_add_int64_v81_910B1"] = dynamic_add_int64_v81;
    opInfoMap_["dynamic_mul_int64_v81_910B1"] = dynamic_mul_int64_v81;
    opInfoMap_["dynamic_maximum_int64_v81_910B1"] = dynamic_maximum_int64_v81;
    opInfoMap_["dynamic_minimum_int64_v81_910B1"] = dynamic_minimum_int64_v81;

    opInfoMap_["dynamic_add_int64_v81_910B2"] = dynamic_add_int64_v81;
    opInfoMap_["dynamic_mul_int64_v81_910B2"] = dynamic_mul_int64_v81;
    opInfoMap_["dynamic_maximum_int64_v81_910B2"] = dynamic_maximum_int64_v81;
    opInfoMap_["dynamic_minimum_int64_v81_910B2"] = dynamic_minimum_int64_v81;

    opInfoMap_["dynamic_add_int64_v81_910B3"] = dynamic_add_int64_v81;
    opInfoMap_["dynamic_mul_int64_v81_910B3"] = dynamic_mul_int64_v81;
    opInfoMap_["dynamic_maximum_int64_v81_910B3"] = dynamic_maximum_int64_v81;
    opInfoMap_["dynamic_minimum_int64_v81_910B3"] = dynamic_minimum_int64_v81;

    opInfoMap_["dynamic_add_int64_v81_910B4"] = dynamic_add_int64_v81;
    opInfoMap_["dynamic_mul_int64_v81_910B4"] = dynamic_mul_int64_v81;
    opInfoMap_["dynamic_maximum_int64_v81_910B4"] = dynamic_maximum_int64_v81;
    opInfoMap_["dynamic_minimum_int64_v81_910B4"] = dynamic_minimum_int64_v81;

    opInfoMap_["dynamic_add_int64_v81_910B4-1"] = dynamic_add_int64_v81;
    opInfoMap_["dynamic_mul_int64_v81_910B4-1"] = dynamic_mul_int64_v81;
    opInfoMap_["dynamic_maximum_int64_v81_910B4-1"] = dynamic_maximum_int64_v81;
    opInfoMap_["dynamic_minimum_int64_v81_910B4-1"] = dynamic_minimum_int64_v81;
}

HcclResult TbeVectorReduce::PrepareVectorReduce(const void *src1, const void *src2, u64 count,
    const HcclDataType dataType, HcclReduceOp redOp, const void *dst, void *stream, void *&addrListDevMemPtr,
    void *&argsHandle, void *&stubFuncAddr, uint32_t &blockDim)
{
    nlohmann::json opDescInfo;
    void *tilingDataDevPtr = nullptr;
    // 获取tiling的执行信息
    TbeReduce::OpRunInfo runInfo;
    CHK_RET(GetTilingDataInfo(count, dataType, redOp, tilingDataDevPtr, opDescInfo, runInfo));

    std::vector<const void *> deviceAddrs;
    CHK_RET(ConcatMem(src1, src2, dst, tilingDataDevPtr, deviceAddrs));
    rtArgsEx_t argsInfo;
    s32 sRet = memset_s(&argsInfo, sizeof(rtArgsEx_t), 0, sizeof(rtArgsEx_t));
    CHK_PRT_RET(sRet != EOK, HCCL_ERROR("[PrepareVectorReduce]memory set error. return[%d].", sRet), HCCL_E_PARA);
    argsInfo.args = deviceAddrs.data();
    argsInfo.argsSize = deviceAddrs.size() * sizeof(void *);
    CHK_RET(hrtGetDevArgsAddr(stream, &argsInfo, &addrListDevMemPtr, &argsHandle));

    s8 *stubFunc = nullptr;
    uint32_t prefetchCnt = 0;
    CHK_RET(GetKernelFunction(dataType, redOp, opDescInfo, runInfo, stubFunc));
    CHK_RET(hrtKernelGetAddrAndPrefCnt(nullptr, 0, reinterpret_cast<void *>(stubFunc), 0, &stubFuncAddr, &prefetchCnt));

    blockDim = runInfo.blockDim;
    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::PrepareTailVectorReduce(void *&devMem1, void *&devMem2, u64 tailCount,
    const HcclDataType dataType, HcclReduceOp redOp, void *stream, void *&addrListDevMemPtr, void *&argsHandle,
    void *&stubFuncAddr, uint32_t &blockDim)
{
    // 申请device内存
    CHK_RET(DevMemMalloc(stream, devMem1, devMem2));

    CHK_RET(PrepareVectorReduce(devMem1, devMem2, tailCount, dataType, redOp, devMem2, stream, addrListDevMemPtr,
        argsHandle, stubFuncAddr, blockDim));

    return HCCL_SUCCESS;
}

HcclResult TbeVectorReduce::JudgeIsTail(const void *src1, const void *src2, const void *dst, u64 count,
    const HcclDataType dataType, u64 &headCount, u64 &tailCount, void *&tailSrc1, void *&tailSrc2, void *&tailDst)
{
    CHK_PRT_RET(!isInit_, HCCL_ERROR("[Run][Reduce]TbeVectorReduce had not been initialized"), HCCL_E_UNAVAIL);
    // 获取总的数据量
    u64 dataSize = SIZE_TABLE[dataType] * count;                                 // 计算总的字节数
    headCount = dataSize / UB_BLOCK_SIZE * UB_BLOCK_SIZE / SIZE_TABLE[dataType]; // 计算出32字节整倍数的数据数量
    tailCount = count - headCount;

    if (tailCount != 0) {
        tailSrc1 = static_cast<char *>(const_cast<void *>(src1)) + (headCount * SIZE_TABLE[dataType]);
        tailSrc2 = static_cast<char *>(const_cast<void *>(src2)) + (headCount * SIZE_TABLE[dataType]);
        tailDst = static_cast<char *>(const_cast<void *>(dst)) + (headCount * SIZE_TABLE[dataType]);
    }
    return HCCL_SUCCESS;
}
} // namespace TbeReduce
